package edu.hawaii.webspider;

import java.io.IOException;
import java.net.MalformedURLException;

import org.xml.sax.SAXException;

import com.meterware.httpunit.GetMethodWebRequest;
import com.meterware.httpunit.WebConversation;
import com.meterware.httpunit.WebRequest;
import com.meterware.httpunit.WebResponse;

/**
 * A demonstration class illustrating how to retrieve and process web pages using HttpUnit.
 * 
 * @author Philip Johnson
 */
public class WebSpiderExample {

  /** Contains the starting url for the crawl. */
  private final String startUrl;
  public int linksFound = 0;

  /**
   * A WebSpider crawls the web and returns info.
   * 
   * @param startUrl The home url for the crawl.
 * @throws SAXException 
 * @throws IOException 
 * @throws MalformedURLException 
   */
  public WebSpiderExample(String startUrl) {
   this.startUrl = startUrl;
  }

  /**
   * Returns the number of links found at the startUrl.
   * 
   * @return The number of links.
   * @throws Exception if problems occur retrieving the startUrl.
   */
  public int getNumLinks() throws Exception {
    // create the conversation object which will maintain state for us
    WebConversation wc = new WebConversation();
    WebRequest request = new GetMethodWebRequest(this.startUrl);
    WebResponse response = wc.getResponse(request);

    System.out.println(wc.getCurrentPage().getURL().toString());
	System.out.println(this.startUrl);

    if (response.getLinks().length > 0) {
        //WebSpiderExample example = new WebSpiderExample("http://www.vanityinkdesigns.com");
        com.meterware.httpunit.WebLink wl[];
        wl = response.getLinks();
        for(int i = 0; i < wl.length; i++)
        {
        	String currentPage = wl[i].getURLString();
        	if (response.isHTML() == true && currentPage != startUrl) {
                search(currentPage);        		
        	}
        	else {
        	search(startUrl + currentPage);
        	}
        	
        	if (checkToManyLinksFound() == true) {
        		break;
        	}
        	//System.out.println(response.getLinks());
        }
        
    }
    
    return response.getLinks().length;
  }

  public boolean checkToManyLinksFound()
  {
	  if(linksFound < 100) return false;
	  return true;
  }
  
  private int search(String currentUrl) {
    WebConversation currentWc = new WebConversation();
    WebRequest currentRequest = new GetMethodWebRequest(currentUrl);
    WebResponse currentResponse;
	try {
		currentResponse = currentWc.getResponse(currentRequest);

    if (currentResponse.getLinks().length > 0) {
        //WebSpiderExample example = new WebSpiderExample("http://www.vanityinkdesigns.com");
        com.meterware.httpunit.WebLink currentWl[];
        currentWl = currentResponse.getLinks();
        
        for(int i = 0; i < currentWl.length; i++) {
        	String currentPage = currentWl[i].getURLString();
        	System.out.println(currentPage);
        	if (currentPage.startsWith("http")) {
                search(currentPage);        		
        	}
        	else {
        	search(currentUrl + currentPage);
        	}
        	
        	//System.out.println(currentResponse.getTitle());
        	linksFound++;
        }
        
    }
	} catch (MalformedURLException e) {
		// TODO Auto-generated catch block
		e.printStackTrace();
	} catch (IOException e) {
		// TODO Auto-generated catch block
		e.printStackTrace();
	} catch (SAXException e) {
		// TODO Auto-generated catch block
		e.printStackTrace();
	}

	  return linksFound;
  }

  /**
   * Retrieves the httpunit home page and counts the number of links on it.
   * 
   * @param args Ignored.
   * @throws Exception If problems occur.
   */
  public static void main(String[] args) throws Exception {
    WebSpiderExample example = new WebSpiderExample("http://www.hackystat.org");
    System.out.println("HttpUnit home page contains " + example.getNumLinks() + " links.");
  }
}
